import cheerio from 'cheerio'
import rp from 'request-promise'
import R from 'ramda'
import _ from 'lodash'
import { writeFileSync } from 'fs'

const sleep = time => new Promise(resolve => setTimeout(resolve,time)) //发动一次休息

let _house = [];
 
export const gethouse = async ( page = 1) =>{
    const options={
        uri:`https://hz.fang.anjuke.com/loupan/all/p${page}/`,
        transform: body => cheerio.load(body),
    }
    console.log("正在爬"+page+ "页");

    const $ = await rp(options)
    let house = [];
    
    $(".key-list .item-mod").each(function(){ //这里不能用箭头函数，会拿不到this
        const link = $(this).attr("link");
        const name = $(this).find(".infos .lp-name .items-name").text();
        const adress =  $(this).find(".address .list-map").text();
        const huxing = $(this).find(".huxing").text();
        const favorPos = $(this).find(".favor-pos .price-txt").text();
        const aroundPrice = $(this).find(".favor-pos .around-price").text();
        house.push({
            link,
            name,
            huxing,
            favorPos,
            aroundPrice,
            adress
        })
    })

    //细化处理
    const fn = R.compose(
        R.map((house) =>{
            const r1 = house.huxing.replace(/\s+/g,""); //去掉空格
            const r2 = house.aroundPrice.replace(/\s+/g,"");
            const reg = /[^\[]*\[(.*)\][^\]]*/; //截取[]里面的数据
            const r3 = house.adress.match(reg);
            const i = house.adress.lastIndexOf("]")+1;
            house.adress = house.adress.slice(i).replace(/\s+/g,"");
            house.huxing = r1;
            house.aroundPrice = r2;
            house.area = r3[1]

            return house
        }),
        R.filter(house => house.name && house.adress && house.huxing && house.favorPos && house.aroundPrice) //判断数据是否齐全，字段不全则省去
    )

        house = fn(house);
        _house = _.union(_house,house)
        
    console.log("已爬到"+_house.length+"条数据")

    if($('.next-page').attr('href')){
        writeFileSync("./server/database/json/house.json",JSON.stringify(_house,null,2),'utf-8')
        await sleep(1000);  
        page++;
        gethouse(page)
    }else{
        console.log("爬完了！")
        return 
    }

}

